#!/usr/bin/env perl

use strict;
use warnings;

BEGIN
{
    unshift(@INC, '.');
    require Data::Dumper;
    require File::Copy;
    require LifeRay;
}

my $dl_path='../../Liferay/DocumentLibrary';

my %mime_types = (
    'doc' => 'application/msword',
    'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
    'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
    'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    'pdf' => 'application/pdf',
    'swf' => 'application/x-shockwave-flash',
    'xls' => 'application/vnd.ms-excel',
    'zip' => 'application/zip'
    );

my @ignored_label_sets = (qw(parents teachers math home), 'Default Tag Set');
my %updated_label_set = (
    map(($_ => "X_$_"), @ignored_label_sets),
    'Country' => 'Origin',
    'Type of Material' => 'Type',
    'Source of material' => 'Source',
    'Grade Level' => 'Grade',
    'Topics' => 'Topic',
    'Regions' => 'Origin'
    );
my %updated_label_name = (
    'primary - all' => 'Primary (All)',
    'secondary - all' => 'Secondary (All)',
    'ministry of education material' => 'Ministry of Education',
    'unesco material' => 'UNESCO',
    'teachers manual' => "Teacher's Manual",
    'dropout or retention programs' => 'Dropout/Retention Programs',
    'multimedia content - audio' => 'Multimedia - Audio',
    'multimedia content - interactive' => 'Multimedia - Interactive',
    'united kingdom' => 'United Kingdom of Great Britain and Northern Ireland',
    'uk and europe' => 'United Kingdom and Europe'
    );

sub parse_labels($)
{
    my ($filename) = @_;
    open(LABELS, "<$filename");
    while (<LABELS>)
    {
	s/[\012\015]*$//;
	my ($set, $name) = /^(.*):(.*)$/;
	$updated_label_name{lc($name)} = $name;
    }
    close(LABELS);
}

sub update_label($)
{
    my ($src) = @_;
    my ($src_set, $src_name) = ($src =~ /^(.*): *(.*)$/);
    die("unknown source set: $src_set")
	if (!exists($updated_label_set{$src_set}));
    my $set = "$updated_label_set{$src_set}";
    return "$set:$src_name" if ($set =~ /^X_/);
    if (!exists($updated_label_name{$src_name}))
    {
	print STDERR "unknown source name: $src_name ($set)\n";
	return "${set}:$src_name";
    }
    return "${set}:$updated_label_name{$src_name}";
}

sub read_folders($$)
{
    my ($lr, $table) = @_;
    print STDERR "Reading folders from $table...\n";
    my @raw = $lr->query("SELECT uuid_, folderId, parentFolderId, name, description FROM $table");
    my %folders = ( 0 => '' );
    while (grep(!exists($folders{$_->{folderId}}), @raw))
    {
	foreach my $child (grep(exists($folders{$_->{parentFolderId}}), @raw))
	{
	    $folders{$child->{folderId}} = $folders{$child->{parentFolderId}}."/".$child->{name};
	}
    }
    return %folders;
}

sub escape_metadata($)
{
    my ($src) = @_;
    return '' if (!defined($src));
    $src =~ s/\n/<div>&nbsp<\/div>/g;
    $src =~ s/"/&#34;/g;
    my @bits = map(ord($_),split(//,$src));
    @bits = map(($_ > 127)?
		map(ord($_), split(//,sprintf("&#%04x;", $_))):
		$_, @bits);
    die($src) if (grep(($_ < 32) || ($_ == 127), @bits));
    return join('',map(chr($_),@bits));
}

sub convert_version($)
{
    my ($version) = @_;
    return $version if ($version eq '1');
    my ($major, $minor) = split(/\./, $version);
    die if ($major != 1);
    return $major + $minor;
}

sub list_versions($$)
{
    my ($dir, $latest) = @_;
    my %versions = ();
    opendir(FILE_DIR, "$dir");
    map(++$versions{$_}, grep($_ ne '.' && $_ ne '..', readdir(FILE_DIR)));
    closedir(FILE_DIR);
    for (my $i = 0; $i < $latest; ++$i)
    {
	++$versions{"1.$i"};
    }
    return keys(%versions);
}

sub read_files($$$)
{
    my ($lr, $folders, $first_id) = @_;
    my @files = ();
    my $bs_id = $first_id;
    print STDERR "Reading digital library files...\n";
    foreach my $file ($lr->query('SELECT uuid_, companyId, groupId, fileEntryId, version, folderId, userId, createDate, modifiedDate, name, title, description FROM dlfileentry'))
    {
	++$bs_id;
	my $id = $file->{fileEntryId};
	my ($ext) = ($file->{name} =~ /\.(.*)$/);	
	$ext =~ tr/A-Z/a-z/;
	print STDERR "warning: mime type for extension $ext unknown\n"
	    if (!exists($mime_types{$ext}));
	my $mime_type = exists($mime_types{$ext})?$mime_types{$ext}:'application/octet-stream';
	my @labels = $lr->fetch_labels('com.liferay.portlet.documentlibrary.model.DLFileEntry', $id);
	@labels = map(update_label($_), @labels);
	my $folder_id = $file->{folderId};
	my %file_base = (
	    # SOURCE => "http://rec.glp.net/c/document_library/get_file?uuid=$file->{uuid_}&groupId=$file->{groupId}",
	    ID => $bs_id,
	    EXTENSION => $ext,
	    TITLE => escape_metadata($file->{title}),
	    AUTHOR => '?',
	    DESCRIPTION => escape_metadata($file->{description}),
	    LABELS => \@labels,
	    FOLDER => $folders->{$folder_id},
	    MIME_TYPE => $mime_type,
	    _TYPE => 'file',
	    _FILENAME => $file->{name}
	    );
	my $file_dir = "${dl_path}/$file->{companyId}/$file->{folderId}/$file->{name}";
	my $latest_version = $file->{version};
	$latest_version = '1.0' if ($latest_version eq '1');
	my @versions = list_versions($file_dir, convert_version($latest_version));
	foreach my $version (@versions)
	{
	    my $bs_version = convert_version($version);
	    my %file_version = %file_base;
	    $file_version{LABELS} = [@{$file_base{LABELS}}];
	    $file_version{SOURCE} = "${file_dir}/${version}";
	    $file_version{VERSION} = $bs_version;
	    $file_version{DESTINATION} = sprintf("glp-rec-library:rec-%06d-%03d", $bs_id, $bs_version),
	    $file_version{_VERSION} = "$version/$latest_version";
	    $file_version{_TITLE} = "$file->{name} ($id/$version)";
	    if ($version eq $latest_version)
	    {
		$file_version{_CREATED} = $file->{createDate};
		$file_version{_MODIFIED} = $file->{modifiedDate};
	    }
	    else
	    {
		push(@{$file_version{LABELS}}, 'Status:Replaced');
	    }
	    push(@{$file_version{LABELS}}, 'Status:Missing')
		if (!-e $file_version{SOURCE});
	    push(@files, \%file_version);
	}
    }
    return @files;
}

sub read_bookmarks($$$)
{
    my ($lr, $folders, $first_id) = @_;
    my @bookmarks = ();
    my $bs_id = $first_id;
    print STDERR "Reading bookmarks...\n";
    foreach my $bookmark ( $lr->query('SELECT uuid_, entryId, folderId, name, url, comments, createDate, modifiedDate FROM bookmarksentry'))
    {
	++$bs_id;
	my $id = $bookmark->{entryId};
	my @labels = $lr->fetch_labels('com.liferay.portlet.bookmarks.model.BookmarksEntry', $id);
	@labels = map(update_label($_), @labels);
	push(@bookmarks, {
	    SOURCE => $bookmark->{url},
	    ID => $bs_id,
	    VERSION => 1,
	    DESTINATION => sprintf("glp-rec-library:rec-%06d-%03d", $bs_id, 1),
	    EXTENSION => 'txt',
	    TITLE => escape_metadata($bookmark->{name}),
	    AUTHOR => '?',
	    DESCRIPTION => escape_metadata($bookmark->{comments}),
	    LABELS => \@labels,
	    FOLDER => $folders->{$bookmark->{folderId}},
	    MIME_TYPE => 'text/plain',
	    _TYPE => 'bookmark',
	    _TITLE => "$bookmark->{uuid_} ($id)",
	    _CREATED => $bookmark->{createDate},
	    _MODIFIED => $bookmark->{modifiedDate}
	     });
    }
    return @bookmarks;
}

sub dump_library(@)
{
    my (@entries) = @_;
    foreach my $entry (@entries)
    {
	print "$entry->{_TITLE}\n";
	print "  SOURCE:      $entry->{SOURCE}\n";
	print "  ID:          $entry->{ID}\n";
	my $lr_version = exists($entry->{_VERSION})?" ($entry->{_VERSION})":'';
	print "  VERSION:     $entry->{VERSION}$lr_version\n";
	print "  DESTINATION: $entry->{DESTINATION} ($entry->{MIME_TYPE})\n";
	print "  FOLDER:      $entry->{FOLDER}\n";
	print "  TITLE:       $entry->{TITLE}\n";
	print "  AUTHOR:      $entry->{AUTHOR}\n";
	my $desc = $entry->{DESCRIPTION} || '-';
	$desc = substr($desc, 0, 37).'...' if (length($desc) > 40);
	print "  DESCRIPTION: $desc\n";
	my @labels = @{$entry->{LABELS}};
	@labels = ('-') if (!@labels);
	print "  LABELS:        $labels[0]\n";
	print join('', map("               $_\n", @labels[1..$#labels]));
    }
}

sub clean_labels(@)
{
    my @items = @_;
    foreach my $item (@items)
    {
	foreach my $set (qw(Topic Grade Type Source Origin))
	{
	    my @labels = grep(/^${set}: /, @{$item->{LABELS}});
	    next if (@labels);
	    print STDERR "warning: $item->{TITLE} has no $set label\n";
	    push(@{$item->{LABELS}}, "$set:Unknown");
	}
    }
}     

sub find_orphan_files($@)
{
    my ($folders, @known) = @_;
    my %known = map(($_->{SOURCE} => 1), @known);
    my @files = ();
    my $bs_id = $known[-1]{ID};
    open(DISK, "ls ${dl_path}/*/*/*/* |");
    while (<DISK>)
    {
	s/[\012\015]*$//;
	my $path = $_;
	next if (exists($known{$path}));
	print STDERR "warning: found orphan file at $path\n";
	my ($folder_id, $base, $ext, $version) = ($path =~ /\/(\d+)\/([^\/]+?)\.([^\/]*?)\/(\d+\.\d+)$/);
	++$bs_id;
	print STDERR "warning: mime type for extension $ext unknown\n"
	    if (!exists($mime_types{$ext}));
	my $mime_type = exists($mime_types{$ext})?$mime_types{$ext}:'application/octet-stream';
	my $bs_version = convert_version($version);
	my $folder = exists($folders->{$folder_id})?$folders->{$folder_id}:'/lost+found';
	push(@files, {
	    ID => $bs_id,
	    EXTENSION => $ext,
	    TITLE => "Orphan file $base.$ext",
	    AUTHOR => '?',
	    DESCRIPTION => '',
	    LABELS => ['Status: Orphan'],
	    FOLDER => $folder,
	    MIME_TYPE => $mime_type,
	    _TYPE => 'file',
	    _FILENAME => "$base.$ext",
	    SOURCE => $path,
	    VERSION => $bs_version,
	    DESTINATION => sprintf("glp-rec-library:rec-%06d-%03d", $bs_id, $bs_version),
	    _VERSION => "$version",
	    _TITLE => "$base.$ext (?/$version)"
	     });
    }
    close(DISK);
    return @files;
}

# Author

my $mode = $ARGV[0] || 'dump';
parse_labels('../doc/README.labels.txt');
my $lr = LifeRay->new();
my %file_folders = read_folders($lr, 'dlfolder');
my @files = read_files($lr, \%file_folders, 10000);
push(@files, find_orphan_files(\%file_folders, @files));
clean_labels(@files);
my %bookmark_folders = read_folders($lr, 'bookmarksfolder');
my @bookmarks = read_bookmarks($lr, \%bookmark_folders, 20000);
clean_labels(@bookmarks);

if ($mode eq 'dump')
{
    dump_library(@files, @bookmarks);
}
elsif ($mode eq 'extract')
{
    my $dest = $ARGV[1] || '../../Liferay/Extracted';
    mkdir($dest) if (!-e $dest);
    die if (!-d $dest);
    foreach my $item (@files, @bookmarks)
    {
	print STDERR "Extracting $item->{DESTINATION}...\n";
	my $header_filename = "$dest/$item->{DESTINATION}.headers.txt";
	open(HEADERS, ">${header_filename}") ||
	    die("unable to open $header_filename for writing");
	print HEADERS "META-GLP-Title: $item->{TITLE}\n";
	print HEADERS "META-GLP-Author: $item->{AUTHOR}\n";
	my $description = $item->{DESCRIPTION} || '-';
	print HEADERS "META-GLP-Description: $description\n";
	print HEADERS "META-GLP-Labels: ".join(', ', grep(!/^X_/, @{$item->{LABELS}}))."\n";
	print HEADERS "META-GLP-Bookmark: $item->{SOURCE}\n"
	    if ($item->{_TYPE} eq 'bookmark');
	print HEADERS "Content-Type: $item->{MIME_TYPE}\n";
	close(HEADERS);
	my $data_filename = "$dest/$item->{DESTINATION}.$item->{EXTENSION}";
	File::Copy::copy($item->{SOURCE}, $data_filename) ||
	    system("touch $data_filename");
    }
}
